# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
import redis
import re


class SnowballMasterPipeline(object):
    def process_item(self, item, spider):
        return item

class MyRedisPipeline(object):
    def __init__(self,host,port):
        self.r = redis.Redis(host=host,port=port,decode_responses=True)

    @classmethod
    def from_crawler(cls,crawler):
        return cls(
            host=crawler.settings.get("REDIS_HOST"),
            port = crawler.settings.get("REDIS_PORT"),
        )

    def process_item(self,item,spider):
        if re.search('/ershoufang/',item['url']):
            self.r.lpush('fangspider:start_urls', item['url'])
        else:
            self.r.lpush('fangspider:no_urls', item['url'])
        # fang_id = re.findall('https://bj.5i5j.com/ershoufang/n(.*?)/',item['url'])
        # if fang_id:
        #     if self.r.sadd("fang:id",fang_id[0]):
        #         self.r.lpush('ershoufang:start_urls',item['url'])
        # else:
        #     self.r.lpush('ershoufang:no_urls',item['url'])